'''
Created on Aug 21, 2012

@author: kingsfield
'''
import os
import re, collections
import ast
import copy

def words(text): return re.findall('[a-z]+', text.lower()) 

def train(features):
    model = collections.defaultdict(lambda: 1)
    for f in features:
        model[f] += 1
    return model

def isCorpus(f, method):
    if method == 'NOCERT':
        return True
    else:
        split = f.split('.')
        if split[-1] == 'txt':
            return True
        else:
            return False

def getCorpus(rootDir, method):
    list_dirs = os.walk(rootDir)
    corpus = ''
    for root, dirs, files in list_dirs: 
        for f in files: 
            if isCorpus(f, method):
                corpus += file(os.path.join(root, f)).read()
    return corpus

#NWORDS = train(words(file('big.txt').read()))
NWORDS = train(words(file('/home/kingsfield/Desktop/BestBuyPc/small_product_data.xml').read()))
#corpus = getCorpus('/home/kingsfield/workspace/cv/dataset/corpus', 'yes')
#corpus += getCorpus('/home/kingsfield/workspace/cv/dataset/corpus_', 'NOCERT')
#print 'get corpus %d' % len(words(corpus))
#NWORDS = train(words(corpus))
#NWORDS = []

alphabet = 'abcdefghijklmnopqrstuvwxyz'

#def edits1(word):
#    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
#    #deletes = [a + b[1:] for a, b in splits if b]
#    #transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b) > 1]
#    replaces = [a + c + b[1:] for a, b in splits for c in alphabet if b and b[0] == 'e']
#    #inserts = [a + c + b for a, b in splits for c in alphabet]
#    #return set(deletes + transposes + replaces + inserts)
#    return set(replaces)

def edits1(word):
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [a + b[1:] for a, b in splits if b and b.isalpha()]
    transposes = [a + b[1] + b[0] + b[2:] for a, b in splits if len(b) > 1]
    replaces = [a + c + b[1:] for a, b in splits for c in alphabet if b and b.isalpha()]
    inserts = [a + c + b     for a, b in splits for c in alphabet]
    return set(deletes + transposes + replaces + inserts)

def known_edits2(word):
    return set(e2 for e1 in edits1(word) for e2 in edits1(e1) if e2 in NWORDS)

def known_editsn(word, n):
    def editsn_warpper(word, n):
        if n != 0:
            sub = editsn_warpper(word, n - 1)
            res = set([e2 for e1 in sub for e2 in edits1(e1)])
            return res
        else:
            return edits1(word)
    candidates = editsn_warpper(word, n - 1)
    return set(candidates)

def known(words): return set(w for w in words if w in NWORDS)

def correct(word):
    #candidates = known([word]) or known(edits1(word)) or known_edits2(word) or [word]
    candidates = known([word]) or known_editsn(word, 4) or [word]
    return max(candidates, key=NWORDS.get)

#words_set = set(['gears'])
words_set = NWORDS
words_dict = collections.defaultdict(lambda: 0)

def add_correct(word, count):
    if word in words_set:
        words_set[word] += count
        return word
    if len(word) <= 4:
        return word
    t = known_editsn(word, 1)
    hit = [w for w in t if w in words_set]
    if len(hit) > 0:
        correct = max(hit, key=words_set.get)
    else:
        t = known_editsn(word, 2)
        hit = [w for w in t if w in words_set]
        if len(hit) > 0:
            correct = max(hit, key=words_set.get)
        else:
            correct = word
    words_set[correct] += count
    return correct


def tail_num(word):
    for idx in xrange(len(word) - 1, 0, -1):
        if word[idx].isalpha():
            break
    w = word[0:idx + 1]
    num = word[idx + 1:] 
    return w, num
            
        

if __name__ == '__main__':
#    print correct('eirebee')
#    print correct('ysql')
#    res = known_editsn('eirebee', 3)
#    print res
#    print 'done'
    with open('/home/kingsfield/Desktop/BestBuyPc/words_spell_test.csv') as fr:
        raw_data = fr.readlines()
    print len(raw_data)
    
    with open('/home/kingsfield/Desktop/BestBuyPc/words_spell_correct_test.csv', 'w') as fw:
        idx = 0
        for line in raw_data:
            idx += 1
            line = line[0:-1]
            words = line.split(' ')
            correct_words = [add_correct(w, 1) for w in words if len(w) >= 3]
#            if len(w) >= 4:
#                ww, num = tail_num(w)
#            else:
#                ww, num = w, ''
#            count = int(count)
#            correct = add_correct(ww, count)
#            outline = '%s %s %s' % (w, correct, num) if len(num) > 0 else '%s %s' % (w, correct)
            outline = ' '.join(correct_words)
            print '%d %s,%s' % (idx, line, outline)
            fw.write(line + ',' + outline + '\n')
            

#    print tail_num('aaaa')
#    print tail_num('aaaa444')
#    print known_edits2('abcd')
    print 'done'
